# -*- coding: utf-8 -*-
"""
Created on 2021-09-15 15:36:03
---------
@summary:
---------
@author: xubin
"""

import feapder
import json
from feapder import Item
from feapder.db.mysqldb import MysqlDB
from feapder.utils.tools import get_md5


class SinoBiologicalSpider(feapder.AirSpider):

    def create_task(self):
        conn = MysqlDB()
        sql = "SELECT id,type,sub_type FROM sinobiological_task_info WHERE sub_type is not NULL"
        data = {}
        for item in conn.find(sql):
            data[item[2]] = item[1]
        return data

    def start_callback(self):

        print("爬虫开始")

    def end_callback(self):
        print("爬虫结束")

    def validate(self, request, response):
        if response.status_code != 200:
            raise Exception("response code not 200")  # 重试

    def start_requests(self):
        self.task_info = self.create_task()
        for key_word, type_ in self.task_info.items():
            data = {
                'rpc': '20',
                'page': '0',
                'TaxonomicTags': key_word
            }
            yield feapder.Request("https://cn.sinobiological.com/SearchResultDetails", data=data, key_word=key_word)

    def parse(self, request, response):
        key_word = request.key_word
        count = response.json['MaxPage']
        for page in range(0, count):
            data = {
                'rpc': '20',
                'page': page,
                'TaxonomicTags': key_word
            }
            yield feapder.Request("https://cn.sinobiological.com/SearchResultDetails", data=data,
                                  callback=self.second_parse, key_word={key_word: page})

    def second_parse(self, request, response):
        info = request.key_word
        for k, p in info.items():
            res = response.json
            item = Item()
            item.table_name = 'temp_json'
            item.id = get_md5(k + str(p))
            item.type = k
            item.page = p
            item.content = json.dumps(res, ensure_ascii=False)
            yield item


if __name__ == "__main__":
    SinoBiologicalSpider(thread_count=4).start()
    pass
